<!DOCTYPE html>
<html lang="en">
  <head>
    <meta charset="utf-8">
    <title>go 并发处理脚本</title>
  
    <meta name="author" content="秀才">

    <!-- Le HTML5 shim, for IE6-8 support of HTML elements -->
    <!--[if lt IE 9]>
      <script src="http://html5shim.googlecode.com/svn/trunk/html5.js"></script>
    <![endif]-->

    <!-- Le styles -->
    <link href="/assets/twitter/stylesheets/bootstrap.min.css" type="text/css" rel="stylesheet" media="all">
<link href="/assets/twitter/stylesheets/style.css" type="text/css" rel="stylesheet" media="all">
 
    <link href="/assets/twitter/widgets/google_prettify/stylesheets/twitter-bootstrap.css" type="text/css" rel="stylesheet" media="all">

    <!-- Le fav and touch icons -->
  <!-- Update these with your own images
    <link rel="shortcut icon" href="images/favicon.ico">
    <link rel="apple-touch-icon" href="images/apple-touch-icon.png">
    <link rel="apple-touch-icon" sizes="72x72" href="images/apple-touch-icon-72x72.png">
    <link rel="apple-touch-icon" sizes="114x114" href="images/apple-touch-icon-114x114.png">
  -->
  </head>

  <body>

    <div class="navbar">
      <div class="navbar-inner">
        <div class="container">
          <a class="brand" href="/">秀才</a>
          <ul class="nav">
            
              


  <li><a href="/tags">标签</a></li>


            
              


  <li><a href="/categories">分类</a></li>


            
              


  <li><a href="/about">关于我</a></li>


            
          </ul>
        </div>
      </div>
    </div>

    <div class="container">
      <div class="content">
        <div class="page-header">
  <h1>go 并发处理脚本 </h1>
</div>

<div class="row">
  <div class="span12">
    <h3>并发处理脚本</h3>

<p>最近经常涉及到脚本的编写。本身项目数据量较大，所以经常编写的脚本需要高并发，干脆就提取出来。</p>

<p>如果有地方用到，只需要实现接口即可。</p>

<p>谨以此文抛砖引玉，不喜勿喷</p>

<pre><code class="language-go">package script

import (
	&quot;fmt&quot;
	&quot;time&quot;
	&quot;errors&quot;
	&quot;flag&quot;
	&quot;pigcome/utils&quot;
)

// 实现此接口即可
type model interface {
  	// 每个goroutine运行的函数
	RunOne(id int64) (params []int64, errInfos map[int64]string, err error)
  	// 最后收集到结果后的处理函数
	HandleResult(res Result)
}

type RunParam struct {
	Start    int64			//起始ID
	End      int64			//结束ID
	Step     int			//同时进行的goroutine数目
	Ids      []int64		//自定义IDs
	Deadline time.Duration	//goroutine超时限制
}

type ResultType int

const (
	_       ResultType = iota
	Success
	Panic
	Timeout
	Error
)

type Result struct {
	Type     ResultType			//结果类型
	ID       int64				//每个goroutine的ID
  	Ress     []int64			//结果的参数(自定义)
	ErrInfos map[int64]string	 //runOne函数自定义的错误信息
	ErrInfo  error				//整个goroutine的错误信息
}

func Run(params *RunParam, m model) (err error) {
	beginAt := time.Now()
	err = CheckParams(params)
	if err != nil {
		return
	}
	// 1000并没有什么特殊意义，只是为了有缓冲，提高速度
	ch := make(chan Result, 1000) 
  	// 令牌，只有持有令牌才能运行，为了控制goroutine同时进行的数目
	token := make(chan struct{}, params.Step)
  	// 以此判断结果是否都处理完成
	done := make(chan struct{})

	go collectResult(params, ch, done, m)
	if len(params.Ids) &gt; 0 {
		for _, id := range params.Ids {
			token &lt;- struct{}{}
			go runOne(params, ch, m, token, id)
		}
	} else {
		for id := params.Start; id &lt; params.End; id++ {
			token &lt;- struct{}{}
			go runOne(params, ch, m, token, id)
		}
	}
	&lt;-done

	since := time.Since(beginAt)
	utils.PrintlnColorful(utils.Yellow, &quot;耗时：&quot;, since.String())
	return
}

func collectResult(params *RunParam, ch chan Result, done chan struct{}, m model) {
	var finishCount, count int64
	idsLength := len(params.Ids)
	if idsLength &gt; 0 {
		count = int64(idsLength)
	} else {
		count = params.End - params.Start
	}
	for {
		result := &lt;-ch
		m.HandleResult(result)
		finishCount++
		if result.ErrInfo != nil {
			utils.PrintfColorful(utils.Red, &quot;%d: %+v\n&quot;, finishCount, result)
		}
		if finishCount &gt;= count {
			done &lt;- struct{}{}
			return
		}
	}
}

func runOne(params *RunParam, ch chan Result, m model, token chan struct{}, id int64) {
	defer func() {
		&lt;-token
		if err := recover(); err != nil {
			ch &lt;- Result{Panic, id, nil, nil, fmt.Errorf(&quot;panic: %v&quot;, err)}
		}
	}()

	type runOneCh struct {
		ress     []int64
		errInfos map[int64]string
		err      error
	}
	errCh := make(chan runOneCh)

	go func(ch chan runOneCh, id int64) {
		ress, errInfos, err := m.RunOne(id)
		errCh &lt;- runOneCh{ress, errInfos, err}
	}(errCh, id)

	select {
	case runOneRes := &lt;-errCh:
		if runOneRes.err != nil {
			ch &lt;- Result{Error, id, nil, nil, runOneRes.err}
		} else {
			ch &lt;- Result{Success, id, runOneRes.ress, nil, nil}
		}
	case &lt;-time.After(params.Deadline):
		ch &lt;- Result{Timeout, id, nil, nil, errors.New(&quot;timeout&quot;)}
	}
}

// 初始化运行参数
func NewParams() (params *RunParam) {
	params = new(RunParam)
	flag.Int64Var(&amp;params.Start, &quot;start&quot;, 0, &quot;&quot;)
	flag.Int64Var(&amp;params.End, &quot;end&quot;, 0, &quot;&quot;)
	flag.IntVar(&amp;params.Step, &quot;step&quot;, 100, &quot;&quot;)
	var second int
	flag.IntVar(&amp;second, &quot;second&quot;, 5, &quot;&quot;)
	flag.Parse()
	params.Deadline = time.Second * time.Duration(second)
	return
}

// 运行参数检测
func CheckParams(params *RunParam) error {
	if params.End &lt;= params.Start {
		return fmt.Errorf(&quot;ress range err, %+v&quot;, *params)
	}
	if params.Deadline &lt;= 0 {
		return fmt.Errorf(&quot;ress deadline err, %+v&quot;, *params)
	}
	return nil
}

</code></pre>

<p>另外，用到了utils的的相关代码，下面贴一下</p>

<pre><code class="language-go">const (
	Red    = &quot;\033[31m&quot;
	Yellow = &quot;\033[33m&quot;
	Green  = &quot;\033[32m&quot;
)

func PrintlnColorful(color string, vals ...interface{}) {
	fmt.Printf(color)
	fmt.Println(vals...)
	fmt.Printf(&quot;\033[0m&quot;)
}

func PrintfColorful(color string, format string, vals ...interface{}) {
	fmt.Printf(color)
	fmt.Printf(format, vals...)
	fmt.Printf(&quot;\033[0m&quot;)
}
</code></pre>

    <hr>
    <div class="pagination">
      <ul>
        <ul>
          
            <li class="prev"><a href="/utils/github%E6%90%9C%E7%B4%A2%E6%8A%80%E5%B7%A7/" title="github搜索技巧">&larr; Previous</a></li>
          
          

            <li><a href="/archive">Archive</a></li>

          
            <li class="next"><a href="/mongodb/mongodb-%E6%85%A2%E6%9F%A5%E8%AF%A2/" title="mongodb 慢查询">Next &rarr;</a></li>
          
          
        </ul>
      </ul>
    </div>
    <hr>
    
  </div>
  
  <!-- <div class="span4">
    <h4>Published</h4>
    <div class="date"><span>2018-01-11</span></div>
    <br>
    <h4>Categories</h4>
    <ul class="tag_box">
    
      <li>
  <a href="/categories/#go-ref">go <span>7</span></a>
</li>
    
    </ul>
    <br>
    <h4>Tags</h4>
    <ul class="tag_box">
    
      <li>
  <a href="/tags/#go-ref">go <span>7</span></a>
</li>
    
      <li>
  <a href="/tags/#%E5%B9%B6%E5%8F%91-ref">并发 <span>6</span></a>
</li>
    
    </ul>
  </div> -->
</div>

      </div>

      <footer>
        <p>&copy; 秀才 2013 
          with help from <a href="http://github.com/wendal/gor" target="_blank" title="Gor -- Fast Blog">Gor</a>
          and <a href="http://twitter.github.com/bootstrap/" target="_blank">Twitter Bootstrap</a>
		  and Idea from <a href="http://ruhoh.com" target="_blank" title="The Definitive Technical Blogging Framework">ruhoh</a>
        </p>
      </footer>

    </div> <!-- /container -->
    <!-- <script src="//cdnjscn.b0.upaiyun.com/libs/prettify/r298/prettify.min.js"></script> -->
    <script src="http://cdn.staticfile.org/prettify/r298/prettify.min.js"></script>
    <script>
      var pres = document.getElementsByTagName("pre");
      for (var i=0; i < pres.length; ++i) {
        pres[i].className = "prettyprint linenums";
      }
      prettyPrint();
    </script>
    
    
  </body>
</html>
